Attack Type Prediction Using Decision Tree, KNN, ANN and K-Means¶

Importing Required Libraries¶

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import confusion_matrix, f1_score, accuracy_score, precision_score, recall_score
from sklearn.feature_selection import mutual_info_classif
from sklearn.tree import export_graphviz
from IPython.display import SVG, display
import graphviz
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from mpl_toolkits.mplot3d import Axes3D

!pip install graphviz

Requirement already satisfied: graphviz in /home/jawad_ahmed/anaconda3/lib/python3.7/site-packages (0.20.1)

dataset = pd.read_csv('Dataset.txt', sep=',')

attack_types = pd.read_csv('Attack_types.txt', sep=' ')

dataset.head()

dataset.tail()

dataset.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 125973 entries, 0 to 125972
Data columns (total 43 columns):
 #   Column                       Non-Null Count   Dtype  
---  ------                       --------------   -----  
 0   duration                     125973 non-null  int64  
 1   protocol_type                125973 non-null  object 
 2   service                      125973 non-null  object 
 3   flag                         125973 non-null  object 
 4   src_bytes                    125973 non-null  int64  
 5   dst_bytes                    125973 non-null  int64  
 6   land                         125973 non-null  int64  
 7   wrong_fragment               125973 non-null  int64  
 8   urgent                       125973 non-null  int64  
 9   hot                          125973 non-null  int64  
 10  num_failed_logins            125973 non-null  int64  
 11  logged_in                    125973 non-null  int64  
 12  num_compromised              125973 non-null  int64  
 13  root_shell                   125973 non-null  int64  
 14  su_attempted                 125973 non-null  int64  
 15  num_root                     125973 non-null  int64  
 16  num_file_creations           125973 non-null  int64  
 17  num_shells                   125973 non-null  int64  
 18  num_access_files             125973 non-null  int64  
 19  num_outbound_cmds            125973 non-null  int64  
 20  is_host_login                125973 non-null  int64  
 21  is_guest_login               125973 non-null  int64  
 22  count                        125973 non-null  int64  
 23  srv_count                    125973 non-null  int64  
 24  serror_rate                  125973 non-null  float64
 25  srv_serror_rate              125973 non-null  float64
 26  rerror_rate                  125973 non-null  float64
 27  srv_rerror_rate              125973 non-null  float64
 28  same_srv_rate                125973 non-null  float64
 29  diff_srv_rate                125973 non-null  float64
 30  srv_diff_host_rate           125973 non-null  float64
 31  dst_host_count               125973 non-null  int64  
 32  dst_host_srv_count           125973 non-null  int64  
 33  dst_host_same_srv_rate       125973 non-null  float64
 34  dst_host_diff_srv_rate       125973 non-null  float64
 35  dst_host_same_src_port_rate  125973 non-null  float64
 36  dst_host_srv_diff_host_rate  125973 non-null  float64
 37  dst_host_serror_rate         125973 non-null  float64
 38  dst_host_srv_serror_rate     125973 non-null  float64
 39  dst_host_rerror_rate         125973 non-null  float64
 40  dst_host_srv_rerror_rate     125973 non-null  float64
 41  attack_category              125973 non-null  object 
 42  occurance                    125973 non-null  int64  
dtypes: float64(15), int64(24), object(4)
memory usage: 41.3+ MB

dataset.describe()

attack_types.head()

attack_types.tail()

attack_types.describe()

attack_types.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 41 entries, 0 to 40
Data columns (total 2 columns):
 #   Column           Non-Null Count  Dtype 
---  ------           --------------  ----- 
 0   attack_category  41 non-null     object
 1   attack_type      41 non-null     object
dtypes: object(2)
memory usage: 784.0+ bytes

Data Visualization¶

# Add a Target Column (Attack Type in the Data Set)
attack_types_dict = attack_types.set_index('attack_category')['attack_type'].to_dict()
target_column_list = []
for attack_category in dataset['attack_category']:
    if (attack_category == 'normal'):
        target_column_list.append('normal')
    else:
        target_column_list.append(attack_types_dict[attack_category])

dataset['attack_type'] = target_column_list

dataset.head()

dataset.tail()

attack_types.head()

attack_types.tail()

attack_types_count = dataset['attack_type'].value_counts()

colors = ['red', 'blue', 'green', 'orange']

fig = plt.figure(figsize=(10,10))
plt.bar(attack_types_count.index, attack_types_count.values, color=colors)

plt.title('Attack Types Frequecies')
plt.xlabel('Attacks')
plt.ylabel('Frequency')

# save the figure as PNG Image
fig.savefig('attack_types_frequencies')

Inference:¶

The graph shows that normal has the highest frequency that mean there are no attacks normally.
Second dos attack has the highest frequency.
Third probe attack has the highest freuqecny.

attack_categories_count = dataset['attack_category'].value_counts()

fig = plt.figure(figsize=(30,20))
plt.bar(attack_categories_count.index, attack_categories_count.values, color=colors)

plt.title('Attack Categories Frequecies')
plt.xlabel('Attack Categories')
plt.ylabel('Frequency')

plt.xticks(rotation=45)
# save the figure as PNG Image
fig.savefig('attack_categories_frequencies')

Inference:¶

The graph shows that normal attack category has the highest frequency.
Second neptune has the highest frequency that correspond the dos attack.
Third satan has the highest frequency that correspond to the probe attack.

Label Encoding the Categorical Columns¶

dataset.head()

categorical_cols = ['protocol_type', 'service', 'flag', 'attack_category', 'attack_type']

label_encoder = LabelEncoder()
for category_col in categorical_cols:
    dataset[category_col] = label_encoder.fit_transform(dataset[category_col])

dataset.head()

Correlation Matrix For Selecting Five Best Features¶

corr_matrix = dataset.corr()
fig, ax = plt.subplots(figsize=(40, 40))
cmap = sns.color_palette("plasma")
sns.heatmap(corr_matrix, cmap=cmap, center=0, square=True, annot=True, annot_kws={'fontsize': 12}, linewidths=.5, ax=ax)
ax.tick_params(axis='both', labelsize=14)
ax.set_title('Correlation Matrix', fontsize=18)
fig.savefig('correlation_matrix_of_dataset.png', dpi=300, bbox_inches='tight')
plt.show()

Inference:¶

The graph shows that attack_type is highly correlated with serror_rate with r = -0.67 negatively correlated.
The graph shows that attack_type is highly correlated with flag with r = 0.41 positively correlated.
The feature 'serror_rate' is higly correlated with 'dst_host_srv_rerror_rate' with r = 0.98 highly positively correlated and with the target column 'serror_rate' r = -0.67 and 'dst_host_srv_rerror_rate' has the r = 0.0082 that mean we will select 'serror_rate' as compared to dst_host_srv_rerror_rate becuase both are higly correlated with each other but with the target column serror_rate is more highly correlated.

corr_matrix = dataset.corr()
corr_target = corr_matrix['attack_type']
top_5_cols = corr_target.abs().sort_values(ascending=False)[1:6]
if 'attack_type' in top_5_cols:
    top_5_cols = top_5_cols.drop(target_var)
top_5_cols_using_corr = top_5_cols.index.tolist()
top_5_cols_using_corr

['dst_host_srv_serror_rate',
 'dst_host_serror_rate',
 'srv_serror_rate',
 'serror_rate',
 'same_srv_rate']

best_feautres_using_corr = dataset[top_5_cols_using_corr]

best_feautres_using_corr.head()

best_feautres_using_corr.describe()

best_feautres_using_corr.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 125973 entries, 0 to 125972
Data columns (total 5 columns):
 #   Column                    Non-Null Count   Dtype  
---  ------                    --------------   -----  
 0   dst_host_srv_serror_rate  125973 non-null  float64
 1   dst_host_serror_rate      125973 non-null  float64
 2   srv_serror_rate           125973 non-null  float64
 3   serror_rate               125973 non-null  float64
 4   same_srv_rate             125973 non-null  float64
dtypes: float64(5)
memory usage: 4.8 MB

corr_matrix = best_feautres_using_corr.corr()
fig, ax = plt.subplots(figsize=(40, 40))
cmap = sns.color_palette("plasma")
sns.heatmap(corr_matrix, cmap=cmap, center=0, square=True, annot=True, annot_kws={'fontsize': 12}, linewidths=.5, ax=ax)
ax.tick_params(axis='both', labelsize=14)
ax.set_title('Correlation Matrix', fontsize=18)
fig.savefig('correlation_matrix_of_features_selected_using_corr.png', dpi=300, bbox_inches='tight')
plt.show()

Applying Decision Tree Model On Features Selected Using Corr Matrix¶

1. Entropy Criteria¶

Split the Data Set into traning and testing dataset¶

X = best_feautres_using_corr
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

X_train.shape

(100778, 5)

X_test.shape

(25195, 5)

y_train.shape

(100778,)

y_test.shape

(25195,)

y_test.head()

378      1
32038    0
86399    0
74412    0
52951    2
Name: attack_type, dtype: int64

y_train.head()

95141    1
37486    0
34926    0
34589    1
11420    1
Name: attack_type, dtype: int64

X_train.head()

X_test.head()

X_test.tail()

X_train.describe()

X_test.describe()

clf_using_corr_entropy = DecisionTreeClassifier(criterion='entropy', random_state=42)
clf_using_corr_entropy.fit(X_train, y_train)

DecisionTreeClassifier(criterion='entropy', random_state=42)

y_pred = clf_using_corr_entropy.predict(X_test)

accuracy = accuracy_score(y_test, y_pred)

print('Accuracy:', accuracy)

Accuracy: 0.8805715419726136

Confusion Matrix¶

cm = confusion_matrix(y_test, y_pred)
# calculate F1 score
f1 = f1_score(y_test, y_pred, average='weighted')
print("F1 Score:", f1)

# calculate accuracy
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)

# calculate error rate
error_rate = 1 - accuracy
print("Error Rate:", error_rate)

# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")

plt.savefig("cm_dt_corr_entropy.png")
plt.show()

F1 Score: 0.8622577711672988
Accuracy: 0.8805715419726136
Error Rate: 0.11942845802738644

Inference:¶

The diagnol shows correctly predicted values. The attack is actually 0 and the model also predicted it as 0 same goes for all the diagnol values like second rows diagnol true value is 1 predicted value is also 1.
The confusion matrix shows that 8.2e + 03 values are predicted correctely True value is 0 and the predicted value is also 0.
Second row first column shows that 1.1e+02 values are predicted wrong True value is 1 but model predicted it as 0.
Same goes for all the confusion matrix.

# Visualize the decision tree
dot_data = export_graphviz(clf_using_corr_entropy, out_file=None, 
                           feature_names=top_5_cols_using_corr,  
                           class_names='attack_type',  
                           filled=True, rounded=True,  
                           special_characters=True)  
# graph = graphviz.Source(dot_data)  
# graph.render("iris_decision_tree")
graph = graphviz.Source(dot_data)
display(SVG(graph.pipe(format='svg')))

2.Gini Criteria¶

X = best_feautres_using_corr
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

clf_corr_using_gini = DecisionTreeClassifier(criterion='gini', random_state=42)
clf_corr_using_gini.fit(X_train, y_train)

DecisionTreeClassifier(random_state=42)

y_pred = clf_corr_using_gini.predict(X_test)

# calculate F1 score
cm = confusion_matrix(y_test, y_pred)
f1 = f1_score(y_test, y_pred, average='weighted')
print("F1 Score:", f1)

# calculate accuracy
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)

# calculate error rate
error_rate = 1 - accuracy
print("Error Rate:", error_rate)

# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_dt_corr_gini.png")
plt.show()

F1 Score: 0.8623194516628205
Accuracy: 0.8806112323873785
Error Rate: 0.11938876761262152

Inference:¶

The diagnol shows correctly predicted values. The attack is actually 0 and the model also predicted it as 0 same goes for all the diagnol values like second rows diagnol true value is 1 predicted value is also 1.
The confusion matrix shows that 8.2e + 03 values are predicted correctely True value is 0 and the predicted value is also 0.
Second row first column shows that 1.1e+02 values are predicted wrong True value is 1 but model predicted it as 0.
Same goes for all the confusion matrix.

# Visualize the decision tree
dot_data = export_graphviz(clf_corr_using_gini, out_file=None, 
                           feature_names=top_5_cols_using_corr,  
                           class_names='attack_type',  
                           filled=True, rounded=True,  
                           special_characters=True)  
# graph = graphviz.Source(dot_data)  
# graph.render("iris_decision_tree")
graph = graphviz.Source(dot_data)
display(SVG(graph.pipe(format='svg')))

Features Selection Using Information Gain¶

# extract the features and target variable
X = dataset.drop('attack_type', axis=1)
y = dataset['attack_type']

# calculate the information gain of each feature
info_gain = mutual_info_classif(X, y)

# create a dataframe to store the information gain of each feature
ig_df = pd.DataFrame({'feature': X.columns, 'info_gain': info_gain})

# sort the dataframe by information gain in descending order
ig_df = ig_df.sort_values(by='info_gain', ascending=False)

# select the top five features based on information gain
top_5_cols_using_ig = list(ig_df['feature'][:5])
print("Top 5 features based on Information Gain:", top_5_cols_using_ig)

Top 5 features based on Information Gain: ['attack_category', 'src_bytes', 'service', 'diff_srv_rate', 'flag']

best_features_using_ig = dataset[top_5_cols_using_ig]

best_features_using_ig.head()

best_features_using_ig.tail()

best_features_using_ig.describe()

best_features_using_ig.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 125973 entries, 0 to 125972
Data columns (total 5 columns):
 #   Column           Non-Null Count   Dtype  
---  ------           --------------   -----  
 0   attack_category  125973 non-null  int64  
 1   src_bytes        125973 non-null  int64  
 2   service          125973 non-null  int64  
 3   diff_srv_rate    125973 non-null  float64
 4   flag             125973 non-null  int64  
dtypes: float64(1), int64(4)
memory usage: 4.8 MB

best_features_using_ig.shape

(125973, 5)

Applying Decision Tree Model Entropy Criteria¶

X = best_features_using_ig
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

X_train.shape

(100778, 5)

X_test.shape

(25195, 5)

y_train.shape

(100778,)

y_test.shape

(25195,)

X_train.head()

X_test.head()

Decision Tree Entropy Criteria¶

clf_using_ig_entropy = DecisionTreeClassifier(criterion='entropy', random_state=42)
clf_using_ig_entropy.fit(X_train, y_train)

DecisionTreeClassifier(criterion='entropy', random_state=42)

Confusion Matrix¶

y_pred = clf_using_ig_entropy.predict(X_test)

# calculate F1 score
cm = confusion_matrix(y_test, y_pred)
f1 = f1_score(y_test, y_pred, average='weighted')
print("F1 Score:", f1)

# calculate accuracy
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)

# calculate error rate
error_rate = 1 - accuracy
print("Error Rate:", error_rate)

# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_dt_ig_entropy.png")
plt.show()

F1 Score: 0.9998534368369423
Accuracy: 0.9998412383409406
Error Rate: 0.0001587616590593699

Inference:¶

The Confusion matrix shows that there are only 4 values that are predicted wrong.
Third row 5th column has a value 1 that shows that True value is 2 but the model predicted it as 4.
Frist row 5th column has a value 3 that shows that True value is 3 but the model predicted it as 4.

# Visualize the decision tree
dot_data = export_graphviz(clf_using_ig_entropy, out_file=None, 
                           feature_names=top_5_cols_using_ig,  
                           class_names='attack_type',  
                           filled=True, rounded=True,  
                           special_characters=True)  
# graph = graphviz.Source(dot_data)  
# graph.render("iris_decision_tree")
graph = graphviz.Source(dot_data)
display(SVG(graph.pipe(format='svg')))

Decision Tree Using Gini Criteria¶

X = best_features_using_ig
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

clf_using_ig_gini = DecisionTreeClassifier(criterion='gini', random_state=42)
clf_using_ig_gini.fit(X_train, y_train)

DecisionTreeClassifier(random_state=42)

Confusion Matrix¶

y_pred = clf_using_ig_gini.predict(X_test)

# calculate F1 score
cm = confusion_matrix(y_test, y_pred)
f1 = f1_score(y_test, y_pred, average='weighted')
print("F1 Score:", f1)

# calculate accuracy
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)

# calculate error rate
error_rate = 1 - accuracy
print("Error Rate:", error_rate)

# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_dt_ig_gini.png")
plt.show()

F1 Score: 0.9998534368369423
Accuracy: 0.9998412383409406
Error Rate: 0.0001587616590593699

# Visualize the decision tree
dot_data = export_graphviz(clf_using_ig_gini, out_file=None, 
                           feature_names=top_5_cols_using_ig,  
                           class_names='attack_type',  
                           filled=True, rounded=True,  
                           special_characters=True)  
# graph = graphviz.Source(dot_data)  
# graph.render("iris_decision_tree")
graph = graphviz.Source(dot_data)
display(SVG(graph.pipe(format='svg')))

Applying Artificial Neural Network Model¶

Applying ANN features selected using Information Gain¶

X = best_features_using_ig
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

X_train.shape

(100778, 5)

X_train.head()

X_test.head()

# Normalize the input features
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

X_train

array([[ 0.14415382, -0.00749435, -0.44179813, -0.34956447,  0.75141773],
       [-0.61684847, -0.00752856,  1.08774919, -0.01525403, -0.73563646],
       [-0.61684847, -0.00752856, -0.44179813, -0.01525403, -2.22269065],
       ...,
       [-0.61684847, -0.00752856, -0.44179813,  0.04046437, -2.22269065],
       [ 0.14415382, -0.00747917, -0.44179813, -0.34956447,  0.75141773],
       [ 0.14415382, -0.00752168, -1.17598084, -0.29384607,  0.75141773]])

X_test

array([[ 0.14415382, -0.0075228 , -1.17598084, -0.34956447,  0.75141773],
       [-0.61684847, -0.00752856, -0.44179813,  0.2076196 , -0.73563646],
       [-0.61684847, -0.00752856,  0.9653854 , -0.01525403, -0.73563646],
       ...,
       [ 0.14415382, -0.00545349, -0.6865257 , -0.34956447,  0.75141773],
       [ 0.14415382, -0.00739862,  1.39365865, -0.34956447,  0.75141773],
       [-0.61684847, -0.00752856,  1.08774919, -0.01525403, -0.73563646]])

y_train.head()

95141    1
37486    0
34926    0
34589    1
11420    1
Name: attack_type, dtype: int64

y_test.head()

378      1
32038    0
86399    0
74412    0
52951    2
Name: attack_type, dtype: int64

model_based_on_ig_features = MLPClassifier(hidden_layer_sizes=(10, 5), activation='relu', solver='adam', max_iter=1000, random_state=1)


model_based_on_ig_features.fit(X_train, y_train)

MLPClassifier(hidden_layer_sizes=(10, 5), max_iter=1000, random_state=1)

y_pred = model_based_on_ig_features.predict(X_test)

cm = confusion_matrix(y_test, y_pred)

sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_ann_ig.png")
plt.show()

Inference:¶

The confusion matrix shows that most the values predicted successfully by the model.
The row 3 column 1 value 17 show that True value is 2 but the model predicted as 0 are 17 values.

# Calculate accuracy, precision, recall, and F1 score
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, average='weighted')
recall = recall_score(y_test, y_pred, average='weighted')
f1 = f1_score(y_test, y_pred, average='weighted')
error_rate = 1 - accuracy

# Print the results
print("Accuracy: {:.3f}".format(accuracy))
print("Precision: {:.3f}".format(precision))
print("Recall: {:.3f}".format(recall))
print("F1 Score: {:.3f}".format(f1))
print("Error Rate: {:.3f}".format(error_rate))

Accuracy: 0.999
Precision: 0.999
Recall: 0.999
F1 Score: 0.999
Error Rate: 0.001

Apply ANN Model on the Features Selected by Correlation Matrix¶

X = best_feautres_using_corr
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Normalize the input features
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

X_train

array([[-0.62456242, -0.63942029, -0.63170499, -0.63715936,  0.77143339],
       [ 1.61966297,  1.60875752,  1.60574115,  1.60288857, -1.48038983],
       [-0.62456242, -0.63942029, -0.63170499, -0.63715936, -1.43489845],
       ...,
       [-0.62456242, -0.63942029, -0.63170499, -0.63715936, -1.45764414],
       [-0.62456242, -0.63942029, -0.63170499, -0.63715936,  0.77143339],
       [-0.62456242, -0.63942029, -0.63170499, -0.63715936,  0.77143339]])

X_test

array([[-0.62456242, -0.63942029, -0.63170499, -0.63715936,  0.77143339],
       [ 1.61966297,  1.60875752,  1.60574115,  1.60288857, -0.9117476 ],
       [ 1.61966297,  1.60875752,  1.60574115,  1.60288857, -1.34391569],
       ...,
       [-0.62456242, -0.63942029, -0.63170499, -0.63715936,  0.77143339],
       [-0.57967791, -0.59445673, -0.63170499, -0.63715936,  0.77143339],
       [ 1.61966297,  1.60875752,  1.60574115,  1.60288857, -1.27567863]])

y_train.head()

95141    1
37486    0
34926    0
34589    1
11420    1
Name: attack_type, dtype: int64

y_test.head()

378      1
32038    0
86399    0
74412    0
52951    2
Name: attack_type, dtype: int64

model_based_on_corr_features = MLPClassifier(hidden_layer_sizes=(10, 5), activation='logistic', solver='adam', max_iter=1000, random_state=1)


model_based_on_corr_features.fit(X_train, y_train)

MLPClassifier(activation='logistic', hidden_layer_sizes=(10, 5), max_iter=1000,
              random_state=1)

y_pred = model_based_on_ig_features.predict(X_test)

cm = confusion_matrix(y_test, y_pred)
# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_ann_corr.png")
plt.show()

Inference:¶

The confusion matrix, F1 score and Error rate shows that features selected based on the Correlation matrix are not good.
They are giving to less accuracy which is not acceptable.

# Calculate accuracy, precision, recall, and F1 score
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, average='weighted')
recall = recall_score(y_test, y_pred, average='weighted')
f1 = f1_score(y_test, y_pred, average='weighted')

# Print the results
print("Accuracy: {:.3f}".format(accuracy))
print("Precision: {:.3f}".format(precision))
print("Recall: {:.3f}".format(recall))
print("F1 Score: {:.3f}".format(f1))

Accuracy: 0.134
Precision: 0.619
Recall: 0.134
F1 Score: 0.116

/home/jawad_ahmed/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

Applying KNN Model¶

Applying KNN on Features Selected By Correlation Matrix¶

X = best_feautres_using_corr
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create a KNN model with k=3
knn = KNeighborsClassifier(n_neighbors=5)

knn.fit(X_train, y_train)

y_pred = knn.predict(X_test)

# Evaluate the model's accuracy
accuracy = knn.score(X_test, y_test)
print("Accuracy:", accuracy)

Accuracy: 0.8744195276840643

Applying Grid Search To Select Best Value of K¶

X = best_feautres_using_corr
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create a KNN model
knn = KNeighborsClassifier()

# Define the grid of hyperparameters to search over
param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11]}

# Create a grid search object
grid_search = GridSearchCV(knn, param_grid, cv=5)

# Train the grid search object using the training set
grid_search.fit(X_train, y_train)

# Print the best parameters and best score
print("Best parameters:", grid_search.best_params_)
print("Best score:", grid_search.best_score_)

Best parameters: {'n_neighbors': 11}
Best score: 0.8057527770613231

knn = KNeighborsClassifier(n_neighbors=11)

# Train the model using the training set
knn.fit(X_train, y_train)

# Make predictions on the testing set
y_pred = knn.predict(X_test)

Confusion Matrix For Performance Evaluation¶

cm = confusion_matrix(y_test, y_pred)
# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_knn_corr.png")
plt.show()

# Calculate accuracy, precision, recall, and F1 score
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, average='weighted')
recall = recall_score(y_test, y_pred, average='weighted')
f1 = f1_score(y_test, y_pred, average='weighted')

# Print the results
print("Accuracy: {:.3f}".format(accuracy))
print("Precision: {:.3f}".format(precision))
print("Recall: {:.3f}".format(recall))
print("F1 Score: {:.3f}".format(f1))

Accuracy: 0.880
Precision: 0.882
Recall: 0.880
F1 Score: 0.862

/home/jawad_ahmed/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

Applying KNN on Features Selected Using Information Gain¶

X = best_features_using_ig
y = dataset['attack_type']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create a KNN model
knn = KNeighborsClassifier()

# Define the grid of hyperparameters to search over
param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11]}

# Create a grid search object
grid_search = GridSearchCV(knn, param_grid, cv=5)

# Train the grid search object using the training set
grid_search.fit(X_train, y_train)

# Print the best parameters and best score
print("Best parameters:", grid_search.best_params_)
print("Best score:", grid_search.best_score_)

Best parameters: {'n_neighbors': 1}
Best score: 0.9989878742385923

knn = KNeighborsClassifier(n_neighbors=1)

# Train the model using the training set
knn.fit(X_train, y_train)

# Make predictions on the testing set
y_pred = knn.predict(X_test)

Confusion Matrix For Performance Evaluation¶

cm = confusion_matrix(y_test, y_pred)
# plot confusion matrix
sns.heatmap(cm, annot=True, cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")
plt.savefig("cm_knn_ig.png")
plt.show()

Inference:¶

The confusion Matrix shows that most of the values are predicted correctly by the model.
Thirty Six values that are not predicted by the model

# Calculate accuracy, precision, recall, and F1 score
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, average='weighted')
recall = recall_score(y_test, y_pred, average='weighted')
f1 = f1_score(y_test, y_pred, average='weighted')

# Print the results
print("Accuracy: {:.3f}".format(accuracy))
print("Precision: {:.3f}".format(precision))
print("Recall: {:.3f}".format(recall))
print("F1 Score: {:.3f}".format(f1))

Accuracy: 0.999
Precision: 0.999
Recall: 0.999
F1 Score: 0.999

Applying K-Mean Clustering Model¶

Features Selected By Correlation Matrix¶

# Create a scaler object
scaler = StandardScaler()

# Fit and transform the data
scaled_data = scaler.fit_transform(best_feautres_using_corr)

sns.set_style('darkgrid')
%config InlineBackend.figure_format = 'retina'

# Visualize the correlation your data and identify variables for further analysis
g = sns.PairGrid(best_feautres_using_corr)
g.map(sns.scatterplot);

Elbow Method¶

fig, ax = plt.subplots(figsize=(20, 15))

# Determine optimal cluster number with elbow method
wcss = []

for i in range(1, 11):
    model = KMeans(n_clusters = i,     
                    init = 'k-means++',                 # Initialization method for kmeans
                    max_iter = 300,                     # Maximum number of iterations 
                    n_init = 10,                        # Choose how often algorithm will run with different centroid 
                    random_state = 0)                   # Choose random state for reproducibility
    model.fit(scaled_data)                              
    wcss.append(model.inertia_)
    
# Show Elbow plot
ax.plot(range(1, 11), wcss)
ax.set_title('Elbow Method')                               # Set plot title
ax.set_xlabel('Number of clusters')                        # Set x axis name
ax.set_ylabel('Within Cluster Sum of Squares (WCSS)')      # Set y axis name

# Save the plot as a PNG image
plt.savefig('elbow_plot.png', dpi=300)

plt.show()

Visualize the Scatter Plot¶

# Convert dataframe to numpy array
data = scaled_data

# Set the number of clusters
k = 4

# Initialize the k-means algorithm
kmeans = KMeans(n_clusters=k, random_state=0)

# Fit the algorithm to the data
labels = kmeans.fit_predict(data)

# Visualize the clusters in 3D
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(data[:, 0], data[:, 1], data[:, 2], c=labels, cmap='rainbow')
ax.set_xlabel('Feature 1')
ax.set_ylabel('Feature 2')
ax.set_zlabel('Feature 3')
plt.title('K-Means Clustering with k=4')

# Save the graph
plt.savefig('kmeans_cluster.png')

# Show the graph
plt.show()

Applying K-Mean On the Feature Selected Using Information Gain¶

best_features_using_ig.head()

# Create a scaler object
scaler = StandardScaler()

# Fit and transform the data
scaled_data = scaler.fit_transform(best_features_using_ig)

# Visualize the correlation your data and identify variables for further analysis
g = sns.PairGrid(best_features_using_ig)
g.map(sns.scatterplot);

Elbow Method¶

fig, ax = plt.subplots(figsize=(20, 15))

# Determine optimal cluster number with elbow method
wcss = []

for i in range(1, 11):
    model = KMeans(n_clusters = i,     
                    init = 'k-means++',                 # Initialization method for kmeans
                    max_iter = 300,                     # Maximum number of iterations 
                    n_init = 10,                        # Choose how often algorithm will run with different centroid 
                    random_state = 0)                   # Choose random state for reproducibility
    model.fit(scaled_data)                              
    wcss.append(model.inertia_)
    
# Show Elbow plot
ax.plot(range(1, 11), wcss)
ax.set_title('Elbow Method')                               # Set plot title
ax.set_xlabel('Number of clusters')                        # Set x axis name
ax.set_ylabel('Within Cluster Sum of Squares (WCSS)')      # Set y axis name

# Save the plot as a PNG image
plt.savefig('elbow_plot.png', dpi=300)

plt.show()

Visualize the Scatter Plot¶

# Convert dataframe to numpy array
data = scaled_data

# Set the number of clusters
k = 8

# Initialize the k-means algorithm
kmeans = KMeans(n_clusters=k, random_state=0)

# Fit the algorithm to the data
labels = kmeans.fit_predict(data)

# Visualize the clusters in 3D
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(data[:, 0], data[:, 1], data[:, 2], c=labels, cmap='rainbow')
ax.set_xlabel('Feature 1')
ax.set_ylabel('Feature 2')
ax.set_zlabel('Feature 3')
plt.title('K-Means Clustering with k=8')

# Save the graph
plt.savefig('kmeans_cluster_1.png')

plt.show()

Select Features Using PCA (Principle Component Analysis)¶

dataset_kmeans = dataset.drop('attack_type', axis=1)

dataset_kmeans.head()

# Instantiate PCA
pca = PCA(n_components=3)

# Fit the PCA to the data
pca.fit(dataset_kmeans)

# Transform the data using the PCA
transformed_data = pca.transform(dataset_kmeans)

# Create a new DataFrame with the transformed data and column names
df_transformed = pd.DataFrame(data=transformed_data, columns=['Feature 1', 'Feature 2', 'Feature 3'])

# Print the explained variance ratio of the PCA
print('Explained variance ratio:', pca.explained_variance_ratio_)

Explained variance ratio: [6.80620901e-01 3.19378965e-01 1.33150173e-07]

df_transformed.head()

# Create a scaler object
scaler = StandardScaler()

# Fit and transform the data
scaled_data = scaler.fit_transform(df_transformed)

scaled_data

array([[-0.00767949, -0.0049157 , -0.10961518],
       [-0.00773826, -0.00491567, -0.10949744],
       [-0.00776313, -0.00491566, -0.10968926],
       ...,
       [-0.00738306, -0.00482032, -0.10956004],
       [-0.00776313, -0.00491566, -0.10969017],
       [-0.00773741, -0.00491567, -0.10962402]])

fig, ax = plt.subplots(figsize=(20, 15))

# Determine optimal cluster number with elbow method
wcss = []

for i in range(1, 11):
    model = KMeans(n_clusters = i,     
                    init = 'k-means++',                 # Initialization method for kmeans
                    max_iter = 300,                     # Maximum number of iterations 
                    n_init = 10,                        # Choose how often algorithm will run with different centroid 
                    random_state = 0)                   # Choose random state for reproducibility
    model.fit(scaled_data)                              
    wcss.append(model.inertia_)
    
# Show Elbow plot
ax.plot(range(1, 11), wcss)
ax.set_title('Elbow Method')                               # Set plot title
ax.set_xlabel('Number of clusters')                        # Set x axis name
ax.set_ylabel('Within Cluster Sum of Squares (WCSS)')      # Set y axis name

# Save the plot as a PNG image
plt.savefig('elbow_plot.png', dpi=300)

plt.show()

# Apply K-means clustering with k=6
kmeans = KMeans(n_clusters=6, random_state=0)
labels = kmeans.fit_predict(scaled_data)

# Visualize the clusters with a scatter plot of three features
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(scaled_data[:, 0], scaled_data[:, 1], scaled_data[:, 2], c=labels, cmap='rainbow')
ax.set_xlabel('Feature 1')
ax.set_ylabel('Feature 2')
ax.set_zlabel('Feature 3')
plt.title('K-Means Clustering with k=6')

# Save the graph
plt.savefig('kmeans_cluster_2.png')

plt.show()

	duration	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	num_failed_logins	logged_in	num_compromised	...	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate	occurance
count	125973.00000	1.259730e+05	1.259730e+05	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	...	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000
mean	287.14465	4.556674e+04	1.977911e+04	0.000198	0.022687	0.000111	0.204409	0.001222	0.395736	0.279250	...	115.653005	0.521242	0.082951	0.148379	0.032542	0.284452	0.278485	0.118832	0.120240	19.504060
std	2604.51531	5.870331e+06	4.021269e+06	0.014086	0.253530	0.014366	2.149968	0.045239	0.489010	23.942042	...	110.702741	0.448949	0.188922	0.308997	0.112564	0.444784	0.445669	0.306557	0.319459	2.291503
min	0.00000	0.000000e+00	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.00000	0.000000e+00	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	10.000000	0.050000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	18.000000
50%	0.00000	4.400000e+01	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	63.000000	0.510000	0.020000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	20.000000
75%	0.00000	2.760000e+02	5.160000e+02	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000	0.000000	...	255.000000	1.000000	0.070000	0.060000	0.020000	1.000000	1.000000	0.000000	0.000000	21.000000
max	42908.00000	1.379964e+09	1.309937e+09	1.000000	3.000000	3.000000	77.000000	5.000000	1.000000	7479.000000	...	255.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	21.000000

	dst_host_srv_serror_rate	dst_host_serror_rate	srv_serror_rate	serror_rate	same_srv_rate
count	125973.000000	125973.000000	125973.000000	125973.000000	125973.000000
mean	0.278485	0.284452	0.282485	0.284485	0.660928
std	0.445669	0.444784	0.447022	0.446456	0.439623
min	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	0.000000	0.000000	0.000000	0.090000
50%	0.000000	0.000000	0.000000	0.000000	1.000000
75%	1.000000	1.000000	1.000000	1.000000	1.000000
max	1.000000	1.000000	1.000000	1.000000	1.000000

	dst_host_srv_serror_rate	dst_host_serror_rate	srv_serror_rate	serror_rate	same_srv_rate
count	100778.000000	100778.000000	100778.000000	100778.000000	100778.000000
mean	0.278298	0.284417	0.282333	0.284440	0.660844
std	0.445590	0.444807	0.446940	0.446421	0.439646
min	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	0.000000	0.000000	0.000000	0.090000
50%	0.000000	0.000000	0.000000	0.000000	1.000000
75%	1.000000	1.000000	1.000000	1.000000	1.000000
max	1.000000	1.000000	1.000000	1.000000	1.000000

	dst_host_srv_serror_rate	dst_host_serror_rate	srv_serror_rate	serror_rate	same_srv_rate
count	25195.000000	25195.000000	25195.000000	25195.000000	25195.000000
mean	0.279232	0.284594	0.283095	0.284662	0.661262
std	0.445993	0.444702	0.447359	0.446602	0.439539
min	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	0.000000	0.000000	0.000000	0.090000
50%	0.000000	0.000000	0.000000	0.000000	1.000000
75%	1.000000	1.000000	1.000000	1.000000	1.000000
max	1.000000	1.000000	1.000000	1.000000	1.000000

	attack_category	src_bytes	service	diff_srv_rate	flag
count	125973.000000	1.259730e+05	125973.000000	125973.000000	125973.000000
mean	10.625483	4.556674e+04	31.226469	0.063053	6.979996
std	2.636084	5.870331e+06	16.346470	0.180314	2.689365
min	0.000000	0.000000e+00	0.000000	0.000000	0.000000
25%	9.000000	0.000000e+00	20.000000	0.000000	5.000000
50%	11.000000	4.400000e+01	24.000000	0.000000	9.000000
75%	11.000000	2.760000e+02	49.000000	0.060000	9.000000
max	22.000000	1.379964e+09	69.000000	1.000000	10.000000

	protocol_type	service	flag	src_bytes	dst_bytes	...	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate	attack_category	occurance
0	tcp	ftp_data	SF	491	0	...	0.17	0.03	0.17	0.00	0.00	0.00	0.05	0.00	normal	20
1	udp	other	SF	146	0	...	0.00	0.60	0.88	0.00	0.00	0.00	0.00	0.00	normal	15
2	tcp	private	S0	0	0	...	0.10	0.05	0.00	0.00	1.00	1.00	0.00	0.00	neptune	19
3	tcp	http	SF	232	8153	...	1.00	0.00	0.03	0.04	0.03	0.01	0.00	0.01	normal	21
4	tcp	http	SF	199	420	...	1.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	normal	21

	duration	protocol_type	service	flag	src_bytes	dst_bytes	...	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	attack_category	occurance
125968	0	tcp	private	S0	0	0	...	0.10	0.06	0.00	1.00	1.0	0.00	neptune	20
125969	8	udp	private	SF	105	145	...	0.96	0.01	0.01	0.00	0.0	0.00	normal	21
125970	0	tcp	smtp	SF	2231	384	...	0.12	0.06	0.00	0.72	0.0	0.01	normal	18
125971	0	tcp	klogin	S0	0	0	...	0.03	0.05	0.00	1.00	1.0	0.00	neptune	20
125972	0	tcp	ftp_data	SF	151	0	...	0.30	0.03	0.30	0.00	0.0	0.00	normal	21

	attack_category	attack_type
0	apache2	dos
1	back	dos
2	mailbomb	dos
3	processtable	dos
4	snmpgetattack	dos

	attack_category	attack_type
36	portsweep	probe
37	satan	probe
38	mscan	probe
39	saint	probe
40	worm	probe

	dst_host_srv_serror_rate	dst_host_serror_rate	srv_serror_rate	serror_rate	same_srv_rate
95141	0.0	0.0	0.0	0.0	1.00
37486	1.0	1.0	1.0	1.0	0.01
34926	0.0	0.0	0.0	0.0	0.03
34589	0.0	0.0	0.0	0.0	1.00
11420	0.0	0.0	0.0	0.0	1.00

	dst_host_srv_serror_rate	dst_host_serror_rate	srv_serror_rate	serror_rate	same_srv_rate
378	0.0	0.00	0.0	0.0	1.00
32038	1.0	1.00	1.0	1.0	0.26
86399	1.0	1.00	1.0	1.0	0.07
74412	0.0	0.00	0.0	0.0	0.07
52951	0.0	0.04	0.0	0.0	1.00

	dst_host_srv_serror_rate	dst_host_serror_rate	srv_serror_rate	serror_rate	same_srv_rate
114267	0.00	0.00	0.0	0.0	1.0
69047	0.00	0.00	0.0	0.0	1.0
56135	0.00	0.00	0.0	0.0	1.0
103927	0.02	0.02	0.0	0.0	1.0
33190	1.00	1.00	1.0	1.0	0.1

	attack_category	src_bytes	service	diff_srv_rate	flag
0	11	491	20	0.00	9
1	11	146	44	0.15	9
2	9	0	49	0.07	5
3	11	232	24	0.00	9
4	11	199	24	0.00	9

	Feature 1	Feature 2	Feature 3
0	-45080.955113	-19767.258863	-284.609780
1	-45425.955109	-19767.168071	-284.304068
2	-45571.955118	-19767.129666	-284.802112
3	-45337.809729	-11614.191028	-286.106906
4	-45372.844627	-19347.182064	-285.608626